<?php
/* tgtai_process
 * This file contains the code to run the batch ai/filter process.  It will retrieve and post for all topics, including all sites
 * in a multisite environment with process_all call.
*/

//add action to set cache duration on simplepie
add_action('wp_feed_options', 'mct_ai_set_simplepie');

//include_once('porterstemmer.php');

define (MCT_AI_OLDPOSTSREAD, '7');
//this should be shorter than the interval in which we run cron, but longer than the longest running time of the process
define (MCT_AI_PIE_CACHE,3600);  
    
function mct_ai_process_all(){
    //set globals
    global $blog_id, $wpdb, $ai_topic_tbl, $ai_postsread_tbl, $ai_sl_pages_tbl, $ai_logs_tbl, $mct_ai_optarray;


    //Loop on all sites
    //Read blogs db and loop
    $blogs[0]['blog_id'] = 1;
    if (is_multisite()) {
        $sql = "SELECT `blog_id`
        FROM $wpdb->blogs";
        $blogs = $wpdb->get_results($sql, ARRAY_A);
    }
    for ($i=0; $i < count($blogs); $i++){
        if (is_multisite()){
           if ($blog_id != $blogs[$i]['blog_id']){
               switch_to_blog($blogs[$i]['blog_id']);
               //Set up tables, options for this blog
               $ai_topic_tbl = $wpdb->prefix.'topic';
               //$ai_postsread_tbl = 'postsread';  Only one master table, so don't change name
               $ai_sl_pages_tbl = $wpdb->prefix.'sl_pages';
               $ai_logs_tbl = $wpdb->prefix.'ai_logs';
               $mct_ai_optarray = get_option('mct_ai_options');
           } 
        }
        mct_ai_process_site();
    } 
    if (is_multisite()){
        restore_current_blog();
        //reset tables globals
        $ai_topic_tbl = $wpdb->prefix.'topic';
        //$ai_postsread_tbl = $wpdb->prefix.'postsread';
        $ai_sl_pages_tbl = $wpdb->prefix.'sl_pages';
        $ai_logs_tbl = $wpdb->prefix.'ai_logs';
        $mct_ai_optarray = get_option('mct_ai_options');

    }
    mct_ai_log('ALL',MCT_AI_LOG_PROCESS, 'Completed ALL Processing  ', '');
}

function mct_ai_process_site(){
    global $blog_id, $wpdb, $ai_topic_tbl, $ai_postsread_tbl, $ai_sl_pages_tbl, $ai_logs_tbl, $mct_ai_optarray;

    mct_ai_log('Blog',MCT_AI_LOG_PROCESS, 'Start Processing Site: '.$blog_id.'  ', '');
    //Clean postsread - also does logs and training targets
    mct_ai_clean_postsread(true);

    //Loop on all train and active topics in this site
    $sql = "SELECT *
            FROM $ai_topic_tbl
            WHERE topic_status != 'Inactive'";
    $topics = $wpdb->get_results($sql, ARRAY_A);
    if (empty($topics)){
        return;
    }
    foreach ($topics as $topic){
        mct_ai_process_topic($topic);
    } //end for each topic
    mct_ai_log('Blog',MCT_AI_LOG_PROCESS, 'End Site Processing: '.$blog_id.'  ', '');
}

function mct_ai_process_topic($topic){ 
    global $blog_id, $wpdb, $ai_topic_tbl, $ai_postsread_tbl, $ai_sl_pages_tbl, $ai_logs_tbl, $mct_ai_optarray;
    //$topic is an array with each field from the topics file
    
    mct_ai_log($topic['topic_name'],MCT_AI_LOG_PROCESS, 'Start Processing Topic ', '');
    if ($topic['topic_type'] == 'Relevance'){
        //instantiate a classifier with topic db's (delete old one)
        $rel = new Relevance();
        $do_relevance = true;
        $dbret = $rel->get_db($topic['topic_name']);
        if ($dbret != ''){
            mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, $dbret, '');  //continue on as classify will return not sure
        } else {
            $rel->preparedb();  //get the sizing right
        }
    }
    //For this topic, get the sources
    $sources = array_map('trim',explode(',',$topic['topic_sources']));
    if (empty($sources)){
        return;
    }
    //Get the restart transient which is set if last topic:source:feed didn't complete
    $restart = get_transient('mct_ai_last_feed');
    if ($restart) {
        $restart_arr = explode(':',$restart);
        if ($topic['topic_id'] != $restart_arr[0]) {
            //not this topic, so skip it
            return;
        } elseif (!in_array($restart_arr[1],$sources)) {
            //Source not in list, sources have changed for this topic, delete the transient & reset variables & go on
            $restart = false;
            $restart_arr = array();
            delete_transient('mct_ai_last_feed');
        }
    }
    foreach ($sources as $source){
        if ($restart && $restart_arr[1] != $source ) continue;  //Not the restart source, so try another
        //For this source, get each feed
        $args = array(
            'category' => $source,
            'orderby' => 'link_id',
            'hide_invisible' => false
        );
        $feeds = get_bookmarks($args);
        if (empty($feeds)){
            continue;
        }

        foreach ($feeds as $feed){
            //Check for restart with this feed
            if ($restart) {
                if ($feed->link_id != $restart_arr[2]) continue;
                //Ok, this is the feed, so lets continue from here, 
                //reset the variables and log that we are restarting
                $restart = false;
                $restart_arr = array();
                mct_ai_log($topic['topic_name'],MCT_AI_LOG_PROCESS, 'Restarting with Feed ', $feed->link_name);
            }
            if (empty($feed->link_rss)){
                mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, 'No feed rss link for feed: '.$feed->link_name, $feed->link_rss);
                continue;  //no feed in link
            }
            //Set the restart transient for 36 hours as we process this feed
            set_transient('mct_ai_last_feed',strval($topic['topic_id']).':'.strval($source).':'.strval($feed->link_id),60*60*36);
            //replace &amp; with & - from db sanitization
            $feed->link_rss = preg_replace('[&amp;]','&',$feed->link_rss);
            //Process Feed
            $nowdate = new DateTime('now');
            $thefeed = fetch_feed($feed->link_rss);
            if (is_wp_error($thefeed)){
                mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, $thefeed->get_error_message(),$feed->link_rss);
                continue;  //feed error
            }
            mct_ai_log($topic['topic_name'],MCT_AI_LOG_PROCESS, 'Processing Feed ', $feed->link_name);
            $anynewposts = false;
            $onlyoldposts = true;
            $cookie = mct_ai_check_feed($feed->link_rss); //Will return a cookie if it had to log in for this feed
            if ($cookie == 'Fail') continue;  //Couldn't log in, so get next feed
            foreach ($thefeed->get_items() as $item){
                //See if old post
                $postdate = new DateTime($item->get_date());
                $postdate->modify('+'.MCT_AI_OLDPOSTSREAD.' day');  //add number of days we keep posts read around
                // $a = $nowdate->diff($postdate,true)->days; for version 5.3 or later only
                if ($postdate < $nowdate){
                    $postdate->modify('+60 day');
                    if ($postdate > $nowdate){
                        $onlyoldposts = false;
                    }
                    continue;  //Post older than what we keep track of
                }
                $anynewposts = true; // Something fits our timeframe
                unset($postdate);
                $post_arr = array();  //will hold the post as we build it
                $page = mct_ai_get_page($item, $topic, $post_arr, $cookie);  //get the page
                if (empty($page)){
                    continue;
                }
                $words = mct_ai_get_words($page, false); //get an array of words, don't remove dups as we need word order for phrase checking
                if (empty($words)){
                    continue;
                }
                $valid = mct_ai_filter_feed($words, $topic, $post_arr); //run through filter
                if (!$valid) {
                    continue;
                }
                if ($topic['topic_tag_search2']){
                    mct_ai_set_tags($words, $topic, $post_arr); //set tags found, including phrases
                }
                //Now remove dups
                $words = mct_ai_no_dups($words);
                 //Do relevance check if appropriate
                if ($do_relevance) {
                    $classed = $rel->classify($words);
                    if ($classed['good'] < 0 && $classed['bad'] < 0){
                        mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'Not trained, unable to classify document',$post_arr['current_link'] );
                    }
                    $post_arr = array_merge($post_arr, $classed);
                }
                mct_ai_post_entry($topic, $post_arr, $page);  //post entries
            
            } //end for each item
            $thefeed->__destruct(); // Do what PHP should be doing on it's own.
            unset($thefeed);
            unset($nowdate);
            //Log if only old posts
            if (!$anynewposts && $onlyoldposts) {
                mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'All posts older than 60 days',$feed->link_name);
            }
        }  //end for each feed
        //Delete the restart transient
        delete_transient('mct_ai_last_feed');
    } //end for each source
    if ($do_relevance){
        //get rid of classifier
        unset($rel);
    }
}  
        

function mct_ai_get_page($item, $topic, &$post_arr, $cookie){
    global $wpdb, $blog_id, $ai_postsread_tbl, $ai_sl_pages_tbl, $mct_ai_stored_page_id, $mct_ai_current_link;
    
    //item is a simplepie object
    //use diffbot to get the page, and store it if not already stored
    
    //clean up the link and check if it is from an excluded domain from this topic
    $ilink = trim($item->get_permalink());
    //replace &amp; with & - from simple pie sanitization
    $ilink = preg_replace('[&amp;]','&',$ilink);
    //If google alert feed, get embedded link
    $cnt = preg_match('{www\.google\.com/url\?q=([^&]*)&ct}',$ilink,$matches);
    if ($cnt) {
        $ilink = trim(rawurldecode($matches[1]));
    }
    //If google news feed, get embedded link
    if (stripos($ilink,'news.google.com/news') !== false){
        $cnt = preg_match('{&url=(.*)$}',$ilink,$matches);
        if ($cnt) {
            $ilink = trim(rawurldecode($matches[1]));
        }
    }
    //If twitter search feed, get embedded links or return '' if none
    if (stripos($ilink,'twitter.com') !== false){
        $desc = $item->get_description();
        $cnt = preg_match('{http://t.co/([^"]*)"}',$desc,$matches);
        if ($cnt) {
            $ilink = trim('http://t.co/'.$matches[1]);  //get just first link
            $ilink = mct_ai_tw_expandurl($ilink);
        } else {
            return '';  //No link found
        }
    }
    //Check if excluded domain for this topic
    $dom_array = explode("\n",$topic['topic_skip_domains']);
    $linkhost = parse_url($ilink, PHP_URL_HOST);  //only check the host/domain as we could get problems with path/query
    foreach ($dom_array as $dom){
        $dom = rtrim($dom);
        if (stripos($linkhost,$dom) !== false) {
            return '';  //excluded domain so skip this item
        }
    }
    $this_topic = trim(strval($blog_id)).':'.trim($topic['topic_id']); // Set up topic key with site and topic
    //check postsread for the item
    //Use double quotes if url has single quote
    if (stripos($ilink,"'") === false){
        $sql = "SELECT `pr_id`, `pr_page_content`, `pr_topics`
            FROM $ai_postsread_tbl
            WHERE pr_url = '$ilink'";
    } else {
        $sql = 'SELECT `pr_id`, `pr_page_content`, `pr_topics`
            FROM '.$ai_postsread_tbl.'
            WHERE pr_url = "$ilink"';
    }
    $pr_row = $wpdb->get_row($sql, ARRAY_A);
    if ($pr_row != NULL){
        //check to see if this topic is in list of topics read
        if (stripos($pr_row['pr_topics'],$this_topic) !== false){
            //We've processed this feed, update date and return blank to signal no more work
            $wpdb->update($ai_postsread_tbl,array('pr_date' => current_time('mysql')), array('pr_id' => $pr_row['pr_id']));
            return '';
        } else {
            // Not processed yet, so get the stored page and update with our topic
            $page_id = $pr_row['pr_id'];
            $pr_row['pr_topics'] .= ','.$this_topic;  
            $upd_array = array(
                'pr_topics' => $pr_row['pr_topics'],
                'pr_date' => current_time('mysql')
            );
            $wpdb->update($ai_postsread_tbl,$upd_array, array('pr_id' => $page_id));  //update postsread with topic/date
            //get stored page
            $page = $pr_row['pr_page_content'];
        }
    } else {
        //Not read yet, so get the page, save it, and update postsread
        $page = mct_ai_getpage($ilink, $topic, $cookie);
        if (empty($page)) {
            //error already logged
            return '';
        }
        //insert into postsread
        $val_array = array(
            'pr_topics' => $this_topic,
            'pr_date' => current_time('mysql'),
            'pr_page_content' => $page,
            'pr_url' => trim($ilink)
        );
        $wpdb->insert($ai_postsread_tbl, $val_array);
        $page_id = $wpdb->insert_id;
        if (!$page_id){
            //log error and return
            mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, 'Could not insert into Postsread DB',$ilink);
            return '';
        }
    }
    $post_arr['current_link'] = $ilink;
    
    return $page;
}


function mct_ai_filter_feed($words, $topic, $post_arr){
    global $wpdb, $mct_ai_current_link, $mct_ai_optarray;
    
    //filter on each topic option - length, keywords
    //Return true if it passes all 
    //remove page if false and return false
    
    if (count($words) < intval($topic['topic_min_length'])){
        mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'Post too short: '.count($words),$post_arr['current_link']);
        return false;
    }
    //exclude keywords, none may be in words  - don't use root search, must match exactly
    if (!empty($topic['topic_exclude'])) {
        $key_str = $topic['topic_exclude'];
        $phrases = mct_ai_pop_phrases($key_str); //remove any phrases
        $key_str = trim($key_str);
        if (!empty($key_str)){
            $keywords = preg_split('{[\s,]+}',$key_str);
            foreach ($keywords as $keyw){
                if (strlen(trim($keyw)) == 0) continue;  //ignore blank keywords from preg problems
                $keyw = strtolower($keyw);
                if (in_array($keyw,$words)) {
                    mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'Found excluded word: '.$keyw,$post_arr['current_link']);
                    return false;
                }
            }
        }
        if (!empty($phrases)){ //check phrases since keywords are ok
            $ret = mct_ai_filterphrase($phrases, $words, 'any');
            if ($ret != 'None'){
                mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'Found excluded phrase: '.$ret,$post_arr['current_link']);
                return false;
            }
        }
    }
    //first keywords, must all be in words
    if (!empty($topic['topic_search_1'])) {
        $key_str = $topic['topic_search_1'];
        $phrases = mct_ai_pop_phrases($key_str); //remove any phrases
        $key_str = trim($key_str);
        if (!empty($key_str)){
            $keywords = preg_split('{[\s,]+}',$key_str);
            foreach ($keywords as $keyw){
                if (strlen(trim($keyw)) == 0) continue;  //ignore blank keywords from preg problems
                $keyw = strtolower($keyw);
                if (!mct_ai_inarray($keyw,$words)) {
                    mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'No Search 1 word: '.$keyw,$post_arr['current_link']);
                    return false;
                }
            }
        }
        if (!empty($phrases)){ //check phrases since keywords are ok
            $ret = mct_ai_filterphrase($phrases, $words, 'all');
            if ($ret != 'Ok'){
                mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'No Search 1 phrase: '.$ret,$post_arr['current_link']);
                return false;
            }
        }
    }
    //LAST CHECK - insert any new before this
    //second keywords must have at least one matching
    if (empty($topic['topic_search_2'])) return true;  //nothing to check so ok
    $key_str = str_replace("\n",' ',$topic['topic_search_2']);
    $phrases = mct_ai_pop_phrases($key_str); //remove any phrases
    $key_str = trim($key_str);
    if (!empty($key_str)){
        $keywords = preg_split('{[\s,]+}',$key_str);
        foreach ($keywords as $keyw){
            $keyw = strtolower($keyw);
            if (strlen(trim($keyw)) == 0) continue;  //ignore blank keywords from preg problems
            if (mct_ai_inarray($keyw,$words)) return true;
        }
    }
    if (!empty($phrases)){ //check phrases since keywords haven't matched
        $ret = mct_ai_filterphrase($phrases, $words, 'any');
        if ($ret != 'None'){
            return true;
        }
    }
    mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'No Search 2 Words',$post_arr['current_link']);
    return false;  //last check wasn't valid  
}

function mct_ai_set_tags($words, $topic, &$post_arr){
    // This function checks each search 2 word/phrase against the document and stores those found in $post_arr[tags]
    if (empty($topic['topic_search_2'])) return true;  //nothing to check so ok
    $key_str = str_replace("\n",' ',$topic['topic_search_2']);
    $phrases = mct_ai_pop_phrases($key_str); //remove any phrases
    $key_str = trim($key_str);
    if (!empty($key_str)){
        $keywords = preg_split('{[\s,]+}',$key_str);
        foreach ($keywords as $keyw){
            if (strlen(trim($keyw)) == 0) continue;  //ignore blank keywords from preg problems
            $keyw = strtolower($keyw);
            if (mct_ai_inarray($keyw,$words)) $post_arr['tags'][] = $keyw;
        }
    }
    if (!empty($phrases)){
        $ret = mct_ai_filterphrase($phrases, $words, 'which');
        if (!empty($ret)) {
            foreach ($ret as $r) {
                $post_arr['tags'][] = $r;
            }
        }
    }
}
function mct_ai_inarray($key, $words){
    //does a simple root word search by finding a word that begins with the key
    foreach($words as $word){
        if (preg_match('{^'.$key.'(.*)$}',$word)){
            return true;
        }
    }
    return false;
}

function mct_ai_pop_phrases(&$keywords){
        //If any phrases are found surrounded by quotes, returns them as an array 
        //and removes them from the keywords argument
        $cnt = preg_match_all('{("|\')([^"\']*)("|\')}',$keywords,$matches);
        if ($cnt) {
            $phrase = array();
            foreach($matches[0] as $match){
                $keywords = preg_replace('{'.$match.'}','',$keywords);
                $match = preg_replace('{("|\')}','',$match);
                $match = trim(strtolower($match));
                $phrase[] = $match;
            }
            return $phrase;
        }
        return '';
    }
    
    function mct_ai_filterphrase($phrase, $words, $type){
        //Checks if $phrase(array) in $words array
        //$type is all - all phrases must be in $words - returns phrase not found or 'Ok'
        // any - any one phrase must be in words - returns phrase found or 'None'
        // which - which phrases are in words (for tag use) - returns an array of phrases found 
        if (empty($phrase)) return;
        if ($type == 'all'){
            foreach ($phrase as $p){
                $pword = preg_split('{[\s,]+}',$p); //get phrase words
                $cnt = count($words);
                $pcnt = count($pword);
                $found_it = false;
                for ($i=0;$i<$cnt;$i++){ //go through each word and match each phrase word sequentially
                    if (preg_match('{^'.$pword[0].'(.*)$}',$words[$i])) {  //use root word matching
                        $found_it = true;  //found first one, will set to false if we don't find all
                        for ($pi=1;$pi<$pcnt;$pi++){
                            if (!preg_match('{^'.$pword[$pi].'(.*)$}',$words[$i+$pi])) {
                                $found_it = false;
                                break;
                            }
                        }
                        if ($found_it) break; //still true, so we are done with this phrase, otherwise keep looking 
                    } 
                }
                if (!$found_it) return $p;  //phrase that wasn't found
            }
            return 'Ok';
        }
        if ($type == 'any'){
            foreach ($phrase as $p){
                $pword = preg_split('{[\s,]+}',$p); //get phrase words
                $cnt = count($words);
                $pcnt = count($pword);
                $found_it = false;
                for ($i=0;$i<$cnt;$i++){ //go through each word and match each phrase word sequentially
                    if (preg_match('{^'.$pword[0].'(.*)$}',$words[$i])) {
                        $found_it = true;  //found first one, will set to false if we don't find all
                        for ($pi=1;$pi<$pcnt;$pi++){
                            if (!preg_match('{^'.$pword[$pi].'(.*)$}',$words[$i+$pi])) {
                                $found_it = false;
                                break;
                            }
                        }
                        if ($found_it) return $p; //still true, so we found a phrase
                    } 
                }
             }
            return 'None'; //should have returned true by now, so must not have found any
        }
        if ($type == 'which'){
            $found_phrases = array();
            foreach ($phrase as $p){
                $pword = preg_split('{[\s,]+}',$p); //get phrase words
                $cnt = count($words);
                $pcnt = count($pword);
                $found_it = false;
                for ($i=0;$i<$cnt;$i++){ //go through each word and match each phrase word sequentially
                    if (preg_match('{^'.$pword[0].'(.*)$}',$words[$i])) {
                        $found_it = true;  //found first one, will set to false if we don't find all
                        for ($pi=1;$pi<$pcnt;$pi++){
                            if (!preg_match('{^'.$pword[$pi].'(.*)$}',$words[$i+$pi])) {
                                $found_it = false;
                                break;
                            }
                        }
                        if ($found_it) {
                            $found_phrases[] = $p; //still true, so we found a phrase
                            break;  //stop looking for this phrase
                        }
                    } 
                }
             }
            return $found_phrases; //return any phrases
        }
        
     }

function mct_ai_post_entry($topic, $post_arr, $page){
    global $wpdb, $ai_sl_pages_tbl, $blog_id, $user_id, $mct_ai_optarray;
    //if filter type just post to blog (cat/tag) or Targets if training
    //else if ai and active: post good to blog(cat/tag), unknown to training
    //  else post training for all 
    //if ai also log ai probs
    
    //Good item, so save a copy of page (postsread version will go away) and setredirect value
    $wpdb->insert($ai_sl_pages_tbl, array ('sl_page_content' => $page));
    $page_id = $wpdb->insert_id;
    if (!$page_id) {
        //log error - couldn't save the page
        mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, 'Could not save page in DB',$post_arr['current_link']);
    } 
    //Set the redirect link
    if (is_multisite()){
        if ($blog_id == 1){
            $link_redir = network_site_url().'blog/'.MCT_AI_REDIR.'/'.trim(strval($page_id));
        } else {
            $link_redir = site_url().'/'.MCT_AI_REDIR.'/'.trim(strval($page_id));
        }
    } else {
        $link_redir = site_url().'/'.MCT_AI_REDIR.'/'.trim(strval($page_id));
    }
    //Set up the content
    mct_ai_getpostcontent($page, $post_arr);
    if ($mct_ai_optarray['ai_show_orig']){
        $post_content = $post_arr['article'].'<br />'.$post_arr['orig_link'];
    } else {
        $post_content = $post_arr['article'].'<p><a href="'.$link_redir.'" >Click here to view full article</a><p>';
    }

    // Get an image if we can - 1st match of appropriate size
    $regexp1 = '{<img [^>]*src\s*=\s*("|\')([^"\']*)("|\')[^>]*>}i'; 
    $pos = preg_match_all($regexp1,$page,$matchall, PREG_SET_ORDER);
    if ($pos) {
        foreach ($matchall as $matches) {
            $size = @getimagesize($matches[2]);
            if ($size && $size[0] >= 25 && $size[1] >= 25){  //excludes small pngs, icons, pixels
                $image = $matches[2];
                break;
            }
        }
    }
    //Set up the values, not set are defaults
    //Check for a user, if not we are in cron process so set an admin user for the site
    $pa = $user_id;
    if (!$pa){
        $useradms = get_users(array('role' => 'administrator'));
        if (empty($useradms)){
            $pa = 1;
        } else {
            $first = $useradms[0];
            $pa = $first->ID;
        }
        wp_set_current_user($pa);
    } 
    $details = array(
      'post_content'  => $post_content,
      'post_author' => $pa,
      'post_title'  =>  $post_arr['title'],
      'ping_status' => 'closed',
      'comment_status' => 'open',
      'post_name' => sanitize_title($post_arr['title']),
      'post_status' => 'publish'
    );
    //Use topic & aiclass in all cases
    if ($topic['topic_type'] == 'Filter'){
        $details['tax_input'] = array (  //add topic name 
        'topic' => $topic['topic_name']
        );
    }
    if ($topic['topic_type'] == 'Relevance'){
        $details['tax_input'] = array (  //add topic name 
        'topic' => $topic['topic_name'],
        'ai_class' => $post_arr['classed'] //add ai class
        );
    }
    //check if active using relevance engine, but post is bad or unknown
    $rel_not_good = false;  
    if ($topic['topic_status'] == 'Active' && 
            $topic['topic_type'] == 'Relevance' && 
            $post_arr['classed'] != 'good'){
        $rel_not_good = true;
    }
    $post_msg = '';
    //Training or not relevant but active
    if ($topic['topic_status'] == 'Training'  || $rel_not_good){
        $post_msg = $post_arr['classed'];
        $details['post_type'] = 'target_ai'; //post as a target
        wp_insert_post($details);
        $posted = $wpdb->last_result;
        if (!empty($post_arr['tags'])){
            $post_id = $posted[0]->ID;
            update_post_meta($post_id,'mct_ai_tag_search2',$post_arr['tags']);
        }
    }
    //Active and relevant
    if ($topic['topic_status'] == 'Active' && !$rel_not_good){
        $post_msg = 'Live';
        if (empty($post_arr['tags'])){
            $tagterm = get_term($topic['topic_tag'],'post_tag');
            $details['tags_input'] = array($tagterm->name);
        } else {
            $details['tags_input'] = $post_arr['tags'];
        }
        $details['post_category'] = array($topic['topic_cat']);
        wp_insert_post($details);
        $posted = $wpdb->last_result;
    }
    
    //update post meta
    $post_id = $posted[0]->ID;
    update_post_meta($post_id,'mct_sl_origurl',array($post_arr['current_link']));
    update_post_meta($post_id,'mct_sl_newurl',array($link_redir));
    //update relevance classification 
    if ($topic['topic_type'] == 'Relevance'){
        update_post_meta($post_id, 'mct_ai_relevance',array(
            'classed' => $post_arr['classed'],
            'good' => sprintf('%.6f',$post_arr['good']),
            'bad' => sprintf('%.6f', $post_arr['bad']),
            'dbsize' => sprintf('%.0f',$post_arr['dbsize'])
        ));
    }
    //update the image if found
    if (!empty($image)){
        update_post_meta($post_id, 'mct_sl_image',$image);
        if ($mct_ai_optarray['ai_alt_theme']) {
            $thumb_id = mct_ai_postthumb($image,$post_id);  //use thumbnail for alt themes
            if ($thumb_id) update_post_meta( $post_id, '_thumbnail_id', $thumb_id );
        }
}
    //update the saved page with the post id
    $wpdb->update($ai_sl_pages_tbl, array('sl_post_id' => $post_id), array ('sl_page_id' => $page_id));
    mct_ai_log($topic['topic_name'],MCT_AI_LOG_ACTIVITY, 'New '.$post_msg.' post',$post_arr['current_link']);
}

function mct_ai_getpostcontent($page, &$post_arr){
    global $mct_ai_optarray;
    
    $excerpt_length = intval(($mct_ai_optarray['ai_excerpt'] > 0) ? $mct_ai_optarray['ai_excerpt'] : 55);
    $title = '';
    $article = '';
    //$page has the content, with html, using the format of diffbot_page function, separate sections
    $cnt = preg_match('{<title>([^<]*)</title>}i',$page,$matches);
    if ($cnt) $title = $matches[1];
    $cnt = preg_match('{<span>(.*)}si',$page,$matches);  //don't stop at end of line
    if ($cnt) $article = $matches[1];
    //Get rid of tags in title
    $title = preg_replace('{<([^>]*)>}',' ',$title);  //remove tags but leave spaces
    $post_arr['title'] = $title;  //save title 
    // Get original URL
    $pos = preg_match('{<div id="source-url">([^>]*)>([^<]*)<}',$page,$matches);
    $post_arr['orig_link'] = $matches[1].'> '.$matches[2].'</a>';
    //Now get article content
    $article = preg_replace('@<style[^>]*>[^<]*</style>@i','',$article);  //remove style tags
    $article = preg_replace('{<([^>]*)>}',' ',$article);  //remove tags but leave spaces
    $article = preg_replace('{&[a-z]*;}',"'",$article);  //remove any encoding
    //Save article snippet
    $excerpt = preg_replace('/\s+/', ' ', $article);  //get rid of extra spaces
    //Get Excerpt of 55 words
    $words = explode(' ', $excerpt, $excerpt_length + 1);
    if ( count($words) > $excerpt_length ) {
            array_pop($words);
            array_push($words, '[...]');
            $excerpt = implode(' ', $words);
    }
    $post_arr['article'] = '<blockquote>'.$excerpt.'</blockquote>';  //save article
    
}

function mct_ai_clean_postsread($pread){
    global $ai_postsread_tbl, $mct_ai_optarray, $wpdb, $ai_logs_tbl;
    
    if ($pread){
        $feed_back = MCT_AI_OLDPOSTSREAD;
        //If not updated since log_days, get rid of saved pages then entries
        $sql = "DELETE FROM $ai_postsread_tbl
                WHERE pr_date < ADDDATE(NOW(),-".$feed_back.")";
        $pr_row = $wpdb->query($sql);
        if (!empty($pr_row)){
            mct_ai_log('Posts Read',MCT_AI_LOG_PROCESS, 'Deleted '.$pr_row, '');
        }
    }
    
    $back = $mct_ai_optarray['ai_log_days'];
    //clean ai_log of errors/activities
    $sql = "DELETE FROM $ai_logs_tbl
            WHERE logs_date < ADDDATE(NOW(),-".$back.")";
    $pr_row = $wpdb->query($sql);
    if (!empty($pr_row)){
        mct_ai_log('Log',MCT_AI_LOG_PROCESS, 'Deleted '.$pr_row, '');
    }
    
    $back = $mct_ai_optarray['ai_train_days'];
    //clean out old training targets, use wp_delete_post which will trigger our hook to delete the saved page
    $postfile = $wpdb->posts;
    $sql = "SELECT ID FROM $postfile WHERE post_type = 'target_ai' AND post_date < ADDDATE(NOW(),-".$back.")";
    $cols = $wpdb->get_col($sql);
    if (!empty($cols)){
        foreach ($cols as $postid){
            wp_delete_post($postid);
        }
        mct_ai_log('Targets',MCT_AI_LOG_PROCESS, 'Deleted '.count($cols), '');
    }
}

function mct_ai_getpage($url, $topic, $cookie){
    
    if ($cookie) {  //need to get page ourselves, then pass to diffbot
        //NOW GET THE PAGE
        $ch = curl_init();
        // SET FILE TO DOWNLOAD
        $agent = "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)";
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_USERAGENT, $agent);
        curl_setopt ($ch, CURLOPT_COOKIEJAR, $cookie);
        curl_setopt ($ch, CURLOPT_COOKIEFILE, $cookie);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
        // EXECUTE REQUEST (FILE DOWNLOAD)
        $content = curl_exec ($ch);
        //check results
        $info = curl_getinfo($ch);
        if ($info['http_code'] != 200) {
            //log error
            mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, 'Curl error: '.$info['http_code'],$url);
            if (empty($info)){
                return '';
            }
        }
        // CLOSE CURL
        curl_close ($ch); 
        return diffbot_post($content, $url);
    }
    
    $dbot = 'http://www.diffbot.com/api/article?token=3b695edfbba1bf45c58c6b7e949355ea&url='.$url.'&html=true';
    // INIT CURL
    $ch = curl_init();
    // SET URL 
    curl_setopt($ch, CURLOPT_URL, $dbot);
    curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,10);
    curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
    //and go
    $content = curl_exec ($ch);
    //check results
    $info = curl_getinfo($ch);
    if ($info['http_code'] != 200) {
        //log error
        mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, 'Curl error: '.$info['http_code'],$url);
        if (empty($info)){
            return '';
        }
    }
    curl_close ($ch); 
    $itm = json_decode($content);
    $page = diffbot_page($itm);
    if (empty($page)) {
        mct_ai_log($topic['topic_name'],MCT_AI_LOG_ERROR, 'Unable to Render Page',$url);
    }
    return $page;
}

function mct_ai_set_simplepie($args){
    $feed = $args;
    $feed->set_cache_duration(MCT_AI_PIE_CACHE);
}

function mct_ai_postthumb($imgurl, $post_id) {
    // Load an image pointed to by a url into the post thumbnail featured image
    // Adapted from code by Aditya Mooley from auto post thumbnail plugin
    

    // Get the file name
    $filename = substr($imgurl, (strrpos($imgurl, '/'))+1);

    if (!(($uploads = wp_upload_dir(current_time('mysql')) ) && false === $uploads['error'])) {
        error_log("Image Uploads Directory Error: ".$uploads['error']);
        return null;
    }

    // Generate unique file name
    $filename = wp_unique_filename( $uploads['path'], $filename );

    // Move the file to the uploads dir
    $new_file = $uploads['path'] . "/$filename";
    
    if (!ini_get('allow_url_fopen')) {
        $file_data = curl_get_file_contents($imgurl);
    } else {
        $file_data = @file_get_contents($imgurl);
    }
    
    if (!$file_data) {
        return null;
    }
    
    $putbytes = file_put_contents($new_file, $file_data);
    //error_log("Image File Put Contents: ".strval($putbytes)." To File: ".$new_file);

    // Set correct file permissions
    $stat = stat( dirname( $new_file ));
    $perms = $stat['mode'] & 0000666;
    @ chmod( $new_file, $perms );

    // Get the file type. Must to use it as a post thumbnail.
    $wp_filetype = wp_check_filetype( $filename );

    extract( $wp_filetype );

    // No file type! No point to proceed further
    if ( ( !$type || !$ext ) && !current_user_can( 'unfiltered_upload' ) ) {
        return null;
    }

    // Compute the URL
    $url = $uploads['url'] . "/$filename";

    // Construct the attachment array
    $attachment = array(
        'post_mime_type' => $type,
        'guid' => $url,
        'post_parent' => null,
        'post_title' => '',
        'post_content' => '',
    );

    $thumb_id = wp_insert_attachment($attachment, false, $post_id);
    if ( !is_wp_error($thumb_id) ) {
        require_once(ABSPATH . '/wp-admin/includes/image.php');
        
        wp_update_attachment_metadata( $thumb_id, wp_generate_attachment_metadata( $thumb_id, $new_file ) );
        update_attached_file( $thumb_id, $new_file );
        return $thumb_id;
    }

    return null;
}

/**
 * Function to fetch the contents of URL using curl in absense of allow_url_fopen.
 * 
 * Copied from user comment on php.net (http://in.php.net/manual/en/function.file-get-contents.php#82255)
 */
function curl_get_file_contents($URL) {
    $c = curl_init();
    curl_setopt($c, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($c, CURLOPT_URL, $URL);
    $contents = curl_exec($c);
    curl_close($c);

    if ($contents) {
        return $contents;
    }
    
    return FALSE;
}
?>